import base64
import io
import time

from io import StringIO, BytesIO

import os
import requests
from PIL import Image
from selenium import webdriver
from browsermobproxy import Server
from selenium.common.exceptions import NoSuchElementException

from selenium.webdriver.chrome.options import Options

ops = {}
ops.setdefault("port",2333)
server = Server(r'E:\Python\Tools\browsermob-proxy-2.1.4\bin\browsermob-proxy.bat',options=ops)
server.start()
proxy = server.create_proxy()

chrome_options = Options()
chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy))

driver = webdriver.Chrome(chrome_options=chrome_options)

# base_url = "https://max.book118.com/html/2018/0522/167985660.shtm"
# base_url = "https://max.book118.com/html/2018/0725/7143031033001141.shtm"
# base_url = "https://max.book118.com/html/2018/1009/7140146015001152.shtm"
base_url = "https://max.book118.com/html/2018/1218/7012166014001165.shtm"
proxy.new_har("douyin", options={'captureHeaders': True, 'captureContent': True})
driver.get(base_url)

# assert '马克思主义基本原理概论' in driver.title
# assert '2018版中国近现代史纲要新教材电子版' in driver.title
look =  driver.find_element_by_id('agree_full')
look.click()
driver.switch_to.frame('layer_new_view_iframe')
time.sleep(3)

index = 1
listlen = len([element for element in proxy.har['log']['entries'] if "wKh2B" in element['request']['url']
               or "wKh2C" in element['request']['url'] or '/img?img=7o@o7x' in element['request']['url']])
while(True):
    try:
        driver.find_element_by_css_selector('[class="icon icon-page-next-disable"]')
        break
    except NoSuchElementException:
        pass
    temp = len([element for element in proxy.har['log']['entries'] if "wKh2B" in element['request']['url']
                or "wKh2C" in element['request']['url'] or '/img?img=7o@o7x' in element['request']['url']])
    print(listlen,"===============",temp)
    if(listlen==2 or listlen < temp):
        _next = driver.find_element_by_css_selector('[class="item page-next"]')
        _next.click()
        listlen = temp
    time.sleep(5)
print("++++++++++start++++++")
result = proxy.har['log']['entries']
for entry in result:
    _url = entry['request']['url']
    # 根据URL找到数据接口
    if "wKh2B" in _url or "wKh2C" in _url or '/img?img=7o@o7x' in _url:
        _url = entry['request']['url']
        response = requests.get(_url)
        image = Image.open(BytesIO(response.content))
        # image.show()
        if image.mode == "P":
            image = image.convert('RGB')
        filepath = 'C:/Users/clq/Desktop/Word/zikao/dd/%s.jpg' % str(index)
        # filepath = 'E:/other/自考专升本/03709马克思基本原理概论/训练试卷/%s.jpg' % str(index)
        image.save(filepath)
        index = index+1
server.stop()
driver.quit()


#抖音爬取
# ops = {}
# ops.setdefault("port",2333)
# server = Server(r'E:\Python\Tools\browsermob-proxy-2.1.4\bin\browsermob-proxy.bat',options=ops)
# server.start()
# proxy = server.create_proxy()
#
# chrome_options = Options()
# chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy))
#
# driver = webdriver.Chrome(chrome_options=chrome_options)
#
# base_url = "https://www.iesdouyin.com/share/user/63174596206"
# proxy.new_har("douyin", options={'captureHeaders': True, 'captureContent': True})
# driver.get(base_url)
#
#
# result = proxy.har
#
# for entry in result['log']['entries']:
#     _url = entry['request']['url']
#     # 根据URL找到数据接口
#     if "/api/v2/aweme/post" in _url:
#         _response = entry['response']
#         _content = _response['content']['text']
#         # 获取接口返回内容
#         print(_content)
#
# server.stop()
# driver.quit()
